<!DOCTYPE html>



  


<html class="theme-next gemini use-motion" lang="zh-Hans">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
<meta name="theme-color" content="#222">









<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />
















  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />







<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.4" rel="stylesheet" type="text/css" />


  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=5.1.4">


  <link rel="mask-icon" href="/images/logo.svg?v=5.1.4" color="#222">





  <meta name="keywords" content="Hexo, NexT" />










<meta name="description" content="python单线程、多线程、多进程爬虫 最近接到一个外包，爬取51jobs上某个地区的所有工作导入excel，正好之前写过一个单线程爬虫爬取51jobs工作岗位的单线程爬虫，进行修改成多线程、多进程，提高爬取的效率….">
<meta property="og:type" content="article">
<meta property="og:title" content="pythond爬虫：高效率">
<meta property="og:url" content="http://example.com/2021/03/17/python%E7%88%AC%E8%99%AB%EF%BC%9A%E9%AB%98%E6%95%88%E7%8E%87/index.html">
<meta property="og:site_name" content="yyc&#39;s blog">
<meta property="og:description" content="python单线程、多线程、多进程爬虫 最近接到一个外包，爬取51jobs上某个地区的所有工作导入excel，正好之前写过一个单线程爬虫爬取51jobs工作岗位的单线程爬虫，进行修改成多线程、多进程，提高爬取的效率….">
<meta property="og:locale">
<meta property="og:image" content="https://gitee.com/goatyyc/giteep-page-imgs/raw/master/imgs/20210317154747.png">
<meta property="og:image" content="https://gitee.com/goatyyc/giteep-page-imgs/raw/master/imgs/20210317154748.png">
<meta property="article:published_time" content="2021-03-17T07:47:44.000Z">
<meta property="article:modified_time" content="2021-03-17T07:49:39.293Z">
<meta property="article:author" content="yyc">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://gitee.com/goatyyc/giteep-page-imgs/raw/master/imgs/20210317154747.png">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '',
    scheme: 'Gemini',
    version: '5.1.4',
    sidebar: {"position":"left","display":"post","offset":12,"b2t":false,"scrollpercent":false,"onmobile":false},
    fancybox: true,
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="http://example.com/2021/03/17/python爬虫：高效率/"/>





  <title>pythond爬虫：高效率 | yyc's blog</title>
  








<meta name="generator" content="Hexo 5.3.0"></head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta custom-logo">
    

    <div class="custom-logo-site-title">
      <a href="/"  class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">yyc's blog</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle">技术博客</p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            关于
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br />
            
            分类
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            归档
          </a>
        </li>
      

      
    </ul>
  

  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://example.com/2021/03/17/python%E7%88%AC%E8%99%AB%EF%BC%9A%E9%AB%98%E6%95%88%E7%8E%87/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.png">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="yyc's blog">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">pythond爬虫：高效率</h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2021-03-17T15:47:44+08:00">
                2021-03-17
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/python/" itemprop="url" rel="index">
                    <span itemprop="name">python</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
          

          
          

          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <h1 id="python单线程、多线程、多进程爬虫"><a href="#python单线程、多线程、多进程爬虫" class="headerlink" title="python单线程、多线程、多进程爬虫"></a>python单线程、多线程、多进程爬虫</h1><blockquote>
<p>最近接到一个外包，爬取51jobs上某个地区的所有工作导入excel，正好之前写过一个单线程爬虫爬取51jobs工作岗位的单线程爬虫，进行修改成多线程、多进程，提高爬取的效率….</p>
</blockquote>
<a id="more"></a>



<h2 id="一、单线程"><a href="#一、单线程" class="headerlink" title="一、单线程"></a>一、单线程</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 导入requests包</span></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line"><span class="keyword">import</span> csv</span><br><span class="line"><span class="keyword">import</span> datetime</span><br><span class="line"></span><br><span class="line"><span class="comment"># 请求头</span></span><br><span class="line">headers = &#123;</span><br><span class="line">        <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36&quot;</span></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取单页信息</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_single_page</span>(<span class="params">url</span>):</span></span><br><span class="line">    res = requests.get(url, headers=headers)</span><br><span class="line">    res.encoding=<span class="string">&#x27;gbk&#x27;</span></span><br><span class="line">    s = res.text</span><br><span class="line"></span><br><span class="line">    selector = etree.HTML(s)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> item <span class="keyword">in</span> selector.xpath(<span class="string">&#x27;/html/body/div[4]/div[2]/div[1]/div/div&#x27;</span>):</span><br><span class="line">        title = item.xpath(<span class="string">&#x27;.//p/span[@class=&quot;title&quot;]/a/text()&#x27;</span>)    <span class="comment"># 职位名称</span></span><br><span class="line">        name = item.xpath(<span class="string">&#x27;.//p/a/@title&#x27;</span>)                          <span class="comment"># 公司名称</span></span><br><span class="line">        location_name = item.xpath(<span class="string">&#x27;.//p/span[@class=&quot;location name&quot;]/text()&#x27;</span>)      <span class="comment"># 地址</span></span><br><span class="line">        sary = item.xpath(<span class="string">&#x27;.//p/span[@class=&quot;location&quot;]/text()&#x27;</span>)                    <span class="comment"># 薪水</span></span><br><span class="line">        time = item.xpath(<span class="string">&#x27;.//p/span[@class=&quot;time&quot;]/text()&#x27;</span>)                        <span class="comment"># 日期</span></span><br><span class="line">        edu = item.xpath(<span class="string">&#x27;.//p[@class=&quot;order&quot;]/text()[1]&#x27;</span>)                          <span class="comment"># 学历</span></span><br><span class="line">        experience = item.xpath(<span class="string">&#x27;.//p[@class=&quot;order&quot;]/text()[2]&#x27;</span>)                          <span class="comment"># 工作经验</span></span><br><span class="line">        properties = item.xpath(<span class="string">&#x27;.//p[@class=&quot;order&quot;]/text()[3]&#x27;</span>)                          <span class="comment"># 公司性质</span></span><br><span class="line">        scale = item.xpath(<span class="string">&#x27;.//p[@class=&quot;order&quot;]/text()[4]&#x27;</span>)                          <span class="comment"># 公司规模</span></span><br><span class="line">        breif = item.xpath(<span class="string">&#x27;.//p[@class=&quot;text&quot;]/@title&#x27;</span>)                          <span class="comment"># 岗位描述</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> <span class="built_in">len</span>(title)&gt;<span class="number">0</span>:</span><br><span class="line">            print(title)</span><br><span class="line">            print(name)</span><br><span class="line">            print(location_name)</span><br><span class="line">            print(sary)</span><br><span class="line">            print(time)</span><br><span class="line">            print(edu)</span><br><span class="line">            print(experience)</span><br><span class="line">            print(properties)</span><br><span class="line">            print(scale)</span><br><span class="line">            print(breif)</span><br><span class="line">            print(<span class="string">&quot;-----------&quot;</span>)</span><br><span class="line"></span><br><span class="line">            <span class="comment"># 存储为csv</span></span><br><span class="line">            <span class="keyword">try</span>:</span><br><span class="line">                <span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">&#x27;data2.csv&#x27;</span>, <span class="string">&#x27;a&#x27;</span>, newline=<span class="string">&#x27;&#x27;</span>, encoding=<span class="string">&#x27;utf-8-sig&#x27;</span>) <span class="keyword">as</span> file_csv:</span><br><span class="line">                        csv_writer = csv.writer(file_csv, dialect=<span class="string">&#x27;excel&#x27;</span>)</span><br><span class="line">                        csv_writer.writerow([title[<span class="number">0</span>], breif[<span class="number">0</span>], name[<span class="number">0</span>], properties[<span class="number">0</span>], <span class="string">&quot;nah&quot;</span>, <span class="string">&quot;nah&quot;</span>, sary[<span class="number">0</span>], location_name[<span class="number">0</span>], <span class="string">&quot;nah&quot;</span>, time[<span class="number">0</span>], scale[<span class="number">0</span>], edu[<span class="number">0</span>], experience[<span class="number">0</span>]])</span><br><span class="line">            <span class="keyword">except</span>:</span><br><span class="line">                <span class="keyword">continue</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 遍历页面</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_pages</span>():</span></span><br><span class="line">    url = <span class="string">&quot;https://jobs.51job.com/shanghai/p&quot;</span></span><br><span class="line"></span><br><span class="line">    page = <span class="number">50</span></span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(page):</span><br><span class="line">        new_url = url+<span class="built_in">str</span>(i+<span class="number">1</span>)+<span class="string">&#x27;/&#x27;</span></span><br><span class="line">        get_single_page(new_url)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    start = datetime.datetime.now()</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 写入csv文件首行</span></span><br><span class="line">    file = <span class="built_in">input</span>(<span class="string">&quot;输入文件名：&quot;</span>)</span><br><span class="line">    <span class="keyword">with</span> <span class="built_in">open</span>(file, <span class="string">&#x27;w&#x27;</span>, newline=<span class="string">&#x27;&#x27;</span>, encoding=<span class="string">&#x27;utf-8-sig&#x27;</span>) <span class="keyword">as</span> file_csv:</span><br><span class="line">        csv_writer = csv.writer(file_csv, dialect=<span class="string">&#x27;excel&#x27;</span>)</span><br><span class="line">        csv_writer.writerow([<span class="string">&quot;职位名称&quot;</span>,<span class="string">&quot;职位详情&quot;</span>,<span class="string">&quot;公司名称&quot;</span>,<span class="string">&quot;公司类型&quot;</span>,<span class="string">&quot;公司主营&quot;</span>,<span class="string">&quot;公司详情&quot;</span>,<span class="string">&quot;薪资&quot;</span>,<span class="string">&quot;工作地点&quot;</span>,<span class="string">&quot;更新日期&quot;</span>,<span class="string">&quot;发布日期&quot;</span>,<span class="string">&quot;公司规模&quot;</span>,<span class="string">&quot;学历要求&quot;</span>,<span class="string">&quot;工作经验&quot;</span>])</span><br><span class="line"></span><br><span class="line">    get_pages()</span><br><span class="line">    <span class="comment"># 爬取50页需要的时间</span></span><br><span class="line">    end = datetime.datetime.now()</span><br><span class="line">    print(<span class="string">&#x27;爬取50页需要的时间 &#x27;</span>+<span class="built_in">str</span>((end - start)))</span><br></pre></td></tr></table></figure>
<p>测试单线程爬取的时间：23s</p>
<p><img src="https://gitee.com/goatyyc/giteep-page-imgs/raw/master/imgs/20210317154747.png" alt="image-20210317140320651"></p>
<h2 id="二、多线程"><a href="#二、多线程" class="headerlink" title="二、多线程"></a>二、多线程</h2><blockquote>
<p>思路：基于单线程爬取的代码，改造成多线程，单线程的代码是获取单页的所有岗位的相关信息遍历写入csv，然后用for循构造用于跳转页面的url，并在该循环中调用获取单页的函数。这就是单线程爬取，每次只能爬取一页，顺序进行爬取。</p>
</blockquote>
<p>python中的多线程：</p>
<h3 id="优点："><a href="#优点：" class="headerlink" title="优点："></a>优点：</h3><p>多线程类似于同时执行多个不同程序，多线程运行有如下优点：</p>
<ul>
<li>使用线程可以把占据长时间的程序中的任务放到后台去处理。</li>
<li>用户界面可以更加吸引人，这样比如用户点击了一个按钮去触发某些事件的处理，可以弹出一个进度条来显示处理的进度</li>
<li>程序的运行速度可能加快</li>
<li>在一些等待的任务实现上如用户输入、文件读写和网络收发数据等，线程就比较有用了。在这种情况下我们可以释放一些珍贵的资源如内存占用等等。</li>
</ul>
<p><strong>python中使用多线程常用的方法：</strong></p>
<p>写一个类继承threading中的Thread类，</p>
<p>（1）重写构造方法<code>__init__</code>初始化多线程类</p>
<p>（2）定义一个run()方法，用于让线程运行指定的任务</p>
<h3 id="示例代码："><a href="#示例代码：" class="headerlink" title="示例代码："></a>示例代码：</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">爬取上海地区51jobs职位数据</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="keyword">from</span> threading <span class="keyword">import</span> Thread</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line"><span class="keyword">import</span> csv</span><br><span class="line"><span class="keyword">from</span> queue <span class="keyword">import</span> Queue </span><br><span class="line"><span class="keyword">import</span> datetime</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Get_jobs</span>(<span class="params">Thread</span>):</span></span><br><span class="line">    <span class="comment"># 重写构造函数</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span>(<span class="params">self, queue</span>):</span></span><br><span class="line">        <span class="comment"># 初始化多线程类的初始化方法</span></span><br><span class="line">        Thread.__init__(self)</span><br><span class="line">        <span class="comment"># 创建类属性</span></span><br><span class="line">        self.queue = queue</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 重写run方法</span></span><br><span class="line">    <span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">    多线程中，通过run()方法让线程运行指定的任务</span></span><br><span class="line"><span class="string">    &#x27;&#x27;&#x27;</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">run</span>(<span class="params">self</span>):</span></span><br><span class="line">        <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">            url = self.queue.get()</span><br><span class="line">            <span class="comment"># print(url)</span></span><br><span class="line">            <span class="keyword">try</span>:</span><br><span class="line">                get_single_page(url)</span><br><span class="line">            <span class="keyword">except</span>:</span><br><span class="line">                print(<span class="string">&quot;fail...&quot;</span>)</span><br><span class="line">            <span class="keyword">finally</span>:</span><br><span class="line">                <span class="comment"># 队列告诉线程任务完成，并且退出</span></span><br><span class="line">                self.queue.task_done()</span><br></pre></td></tr></table></figure>


<p><strong>调用多线程的方法：</strong></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 入口函数</span></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    file = <span class="built_in">input</span>()</span><br><span class="line">    start = datetime.datetime.now()</span><br><span class="line">    _url = <span class="string">&#x27;https://jobs.51job.com/shanghai/p&#123;page&#125;&#x27;</span></span><br><span class="line">    urls = [_url.<span class="built_in">format</span>(page=page) <span class="keyword">for</span> page <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>,<span class="number">50</span>)]</span><br><span class="line"></span><br><span class="line">    queue = Queue()</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 创建线程</span></span><br><span class="line">    <span class="keyword">for</span> x <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">        worker = Get_jobs(queue)</span><br><span class="line">        <span class="comment"># 设定当前线程为守护线程</span></span><br><span class="line">        worker.daemon = <span class="literal">True</span></span><br><span class="line">        worker.start()</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 将网站链接压入队列</span></span><br><span class="line">    <span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line">        queue.put(url)</span><br><span class="line">    </span><br><span class="line">    <span class="comment"># 等队列为空，则退出爬虫</span></span><br><span class="line">    queue.join()</span><br><span class="line">    <span class="comment"># print(&#x27;done...&#x27;) </span></span><br><span class="line">    end = datetime.datetime.now()</span><br><span class="line"></span><br><span class="line">    print(<span class="string">&quot;多线程爬取50页需要：&quot;</span>+<span class="built_in">str</span>((end-start)))</span><br></pre></td></tr></table></figure>
<p><strong>再补充一下调用爬取单页的函数：</strong></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 获取单页信息</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_single_page</span>(<span class="params">url</span>):</span></span><br><span class="line">    headers = &#123;</span><br><span class="line">        <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36&quot;</span></span><br><span class="line">    &#125;</span><br><span class="line">    res = requests.get(url, headers=headers)</span><br><span class="line">    res.encoding=<span class="string">&#x27;gbk&#x27;</span></span><br><span class="line">    s = res.text</span><br><span class="line"></span><br><span class="line">    selector = etree.HTML(s)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> item <span class="keyword">in</span> selector.xpath(<span class="string">&#x27;/html/body/div[4]/div[2]/div[1]/div/div&#x27;</span>):</span><br><span class="line">        title = item.xpath(<span class="string">&#x27;.//p/span[@class=&quot;title&quot;]/a/text()&#x27;</span>)    <span class="comment"># 职位名称</span></span><br><span class="line">        name = item.xpath(<span class="string">&#x27;.//p/a/@title&#x27;</span>)                          <span class="comment"># 公司名称</span></span><br><span class="line">        location_name = item.xpath(<span class="string">&#x27;.//p/span[@class=&quot;location name&quot;]/text()&#x27;</span>)      <span class="comment"># 地址</span></span><br><span class="line">        sary = item.xpath(<span class="string">&#x27;.//p/span[@class=&quot;location&quot;]/text()&#x27;</span>)                    <span class="comment"># 薪水</span></span><br><span class="line">        time = item.xpath(<span class="string">&#x27;.//p/span[@class=&quot;time&quot;]/text()&#x27;</span>)                        <span class="comment"># 日期</span></span><br><span class="line">        edu = item.xpath(<span class="string">&#x27;.//p[@class=&quot;order&quot;]/text()[1]&#x27;</span>)                          <span class="comment"># 学历</span></span><br><span class="line">        experience = item.xpath(<span class="string">&#x27;.//p[@class=&quot;order&quot;]/text()[2]&#x27;</span>)                          <span class="comment"># 工作经验</span></span><br><span class="line">        properties = item.xpath(<span class="string">&#x27;.//p[@class=&quot;order&quot;]/text()[3]&#x27;</span>)                          <span class="comment"># 公司性质</span></span><br><span class="line">        scale = item.xpath(<span class="string">&#x27;.//p[@class=&quot;order&quot;]/text()[4]&#x27;</span>)                          <span class="comment"># 公司规模</span></span><br><span class="line">        breif = item.xpath(<span class="string">&#x27;.//p[@class=&quot;text&quot;]/@title&#x27;</span>)                          <span class="comment"># 岗位描述</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> <span class="built_in">len</span>(title)&gt;<span class="number">0</span>:</span><br><span class="line">            <span class="comment"># print(title)</span></span><br><span class="line">            <span class="comment"># print(name)</span></span><br><span class="line">            <span class="comment"># print(location_name)</span></span><br><span class="line">            <span class="comment"># print(sary)</span></span><br><span class="line">            <span class="comment"># print(time)</span></span><br><span class="line">            <span class="comment"># print(edu)</span></span><br><span class="line">            <span class="comment"># print(experience)</span></span><br><span class="line">            <span class="comment"># print(properties)</span></span><br><span class="line">            <span class="comment"># print(scale)</span></span><br><span class="line">            <span class="comment"># print(breif)</span></span><br><span class="line">            <span class="comment"># print(&quot;-----------&quot;)</span></span><br><span class="line"></span><br><span class="line">            <span class="comment"># 存储为csv</span></span><br><span class="line">            <span class="keyword">try</span>:</span><br><span class="line">                <span class="keyword">with</span> <span class="built_in">open</span>(file, <span class="string">&#x27;a+&#x27;</span>, newline=<span class="string">&#x27;&#x27;</span>, encoding=<span class="string">&#x27;utf-8-sig&#x27;</span>) <span class="keyword">as</span> file_csv:</span><br><span class="line">                        csv_writer = csv.writer(file_csv, dialect=<span class="string">&#x27;excel&#x27;</span>)</span><br><span class="line">                        csv_writer.writerow([title[<span class="number">0</span>], breif[<span class="number">0</span>], name[<span class="number">0</span>], properties[<span class="number">0</span>], <span class="string">&quot;nah&quot;</span>, <span class="string">&quot;nah&quot;</span>, sary[<span class="number">0</span>], location_name[<span class="number">0</span>], <span class="string">&quot;nah&quot;</span>, time[<span class="number">0</span>], scale[<span class="number">0</span>], edu[<span class="number">0</span>], experience[<span class="number">0</span>]])</span><br><span class="line">            <span class="keyword">except</span>:</span><br><span class="line">                <span class="keyword">continue</span></span><br></pre></td></tr></table></figure>


<h3 id="运行结果："><a href="#运行结果：" class="headerlink" title="运行结果："></a>运行结果：</h3><p><img src="https://gitee.com/goatyyc/giteep-page-imgs/raw/master/imgs/20210317154748.png" alt="image-20210317142050834"></p>
<p>爬取50页的数据仅仅耗时3秒，相比于单线程的23秒快了近8倍</p>
<blockquote>
<p> 因为GIL（全局解释器锁）的限制（GIL是用来保证在任意时刻只能有一个控制线程在执行），所以python中的多线程并非真正的多线程。只有 python程序是I/O密集型应用时（I/O的执行：比如读取内存、磁盘、网络），多线程才会对运行效率有显著提高（因在等待I/O的时，会释放GIL允许其他线程继续执行），而在计算密集型应用 中，多线程并没有什么用处。而案例中，正好是因为每页刚好有我们需要的资源进行整页面相关数据的爬取，这样比较耗时，因此在I/O等待时间，开启多线程提高了爬取的效率。</p>
</blockquote>
<p>多线程：threading，就是利用CPU与IO可以同时执行的原因，让CPU在IO等待期间充分运行</p>
<h2 id="三、多进程"><a href="#三、多进程" class="headerlink" title="三、多进程"></a>三、多进程</h2><blockquote>
<p>在python爬虫中，一般用的是多进程进行爬取，因为多线程并不能提高CPU的使用率，而且多线程其实是交替执行，多进程才是并发执行。</p>
</blockquote>
<p>参考一个简单教程： <a target="_blank" rel="noopener" href="https://www.jianshu.com/p/7463bcca9ad5">https://www.jianshu.com/p/7463bcca9ad5</a></p>
<p>与多线程区别开，主要的问题是如何解决多进程之间的通信问题，可以队列</p>
<p>但是这里业务比较简单，不需要进行通信，所以大致的思路和多线程相同，</p>
<p>把url存入先进先出的队列，然后创建多个线程并行执行即可</p>
<h3 id=""><a href="#" class="headerlink" title=""></a></h3>
      
    </div>
    
    
    

    

    

    

    <footer class="post-footer">
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2021/03/16/springboot-%E4%BA%8C%E6%AC%A1%E5%BC%80%E5%8F%91blog%EF%BC%88%E4%BA%8C%EF%BC%89/" rel="next" title="springboot 二次开发blog（二）">
                <i class="fa fa-chevron-left"></i> springboot 二次开发blog（二）
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2021/03/20/mybatis-%E6%93%8D%E4%BD%9C%E7%AF%87%EF%BC%88%E4%B8%80%EF%BC%89/" rel="prev" title="mybatis 操作篇（一）">
                mybatis 操作篇（一） <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>



    <div class="post-spread">
      
    </div>
  </div>


          </div>
          


          

  



        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview-wrap sidebar-panel">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
            
              <img class="site-author-image" itemprop="image"
                src="/images/avatar.png"
                alt="" />
            
              <p class="site-author-name" itemprop="name"></p>
              <p class="site-description motion-element" itemprop="description"></p>
          </div>

          <nav class="site-state motion-element">

            
              <div class="site-state-item site-state-posts">
                <a href="/archives">
                  <span class="site-state-item-count">23</span>
                  <span class="site-state-item-name">日志</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-categories">
                <a href="/categories/index.html">
                  <span class="site-state-item-count">9</span>
                  <span class="site-state-item-name">分类</span>
                </a>
              </div>
            

            

          </nav>

          

          

          
          

          
          

          

        </div>
      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#python%E5%8D%95%E7%BA%BF%E7%A8%8B%E3%80%81%E5%A4%9A%E7%BA%BF%E7%A8%8B%E3%80%81%E5%A4%9A%E8%BF%9B%E7%A8%8B%E7%88%AC%E8%99%AB"><span class="nav-text">python单线程、多线程、多进程爬虫</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#%E4%B8%80%E3%80%81%E5%8D%95%E7%BA%BF%E7%A8%8B"><span class="nav-text">一、单线程</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E4%BA%8C%E3%80%81%E5%A4%9A%E7%BA%BF%E7%A8%8B"><span class="nav-text">二、多线程</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#%E4%BC%98%E7%82%B9%EF%BC%9A"><span class="nav-text">优点：</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E7%A4%BA%E4%BE%8B%E4%BB%A3%E7%A0%81%EF%BC%9A"><span class="nav-text">示例代码：</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E8%BF%90%E8%A1%8C%E7%BB%93%E6%9E%9C%EF%BC%9A"><span class="nav-text">运行结果：</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E4%B8%89%E3%80%81%E5%A4%9A%E8%BF%9B%E7%A8%8B"><span class="nav-text">三、多进程</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link"><span class="nav-text"></span></a></li></ol></li></ol></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">yyc</span>

  
</div>


  <div class="powered-by">由 <a class="theme-link" target="_blank" href="https://hexo.io">Hexo</a> 强力驱动</div>



  <span class="post-meta-divider">|</span>



  <div class="theme-info">主题 &mdash; <a class="theme-link" target="_blank" href="https://github.com/iissnan/hexo-theme-next">NexT.Gemini</a> v5.1.4</div>




        







        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  


  











  
  
    <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>
  

  
  
    <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>
  

  
  
    <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>
  

  
  
    <script type="text/javascript" src="/lib/canvas-nest/canvas-nest.min.js"></script>
  


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.4"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.4"></script>



  
  


  <script type="text/javascript" src="/js/src/affix.js?v=5.1.4"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.1.4"></script>



  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.4"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.4"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.4"></script>



  


  




	





  





  












  





  

  

  

  
  

  

  

  

</body>
</html>
